package com.eric.r2d2.linking;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

public class GenerateLinkingKV {

	public static void main(String[] args) throws IOException {
		
			File f = new File(args[0]);
			BufferedReader br = new BufferedReader(new FileReader(f));
			BufferedWriter bw = new BufferedWriter(new FileWriter(new File("LinkingKV_"+f.getName())));
			
			String line;
			while ((line = br.readLine()) != null) {
				JSONObject j = (JSONObject) JSON.parse(line);
				JSONObject kv = new JSONObject();
				
				List<Object> links = j.getJSONArray("links");
				
				for (Object o : links) {
					//Linking info
					kv.put("linking_url", j.getString("url"));
					kv.put("linking_url_hash", new SimHash(j.getString("url"),64).simHash());
					kv.put("linking_title", j.getJSONObject("title").getString("content"));
					kv.put("linking_domain", j.getString("domain"));
					kv.put("linking_root", j.getString("root_domain"));
					kv.put("link_found", j.getString("crawl_date"));
					
					//Linked info
					kv.put("linked_url", ((JSONObject)o).getString("url"));
					kv.put("linked_url_hash", new SimHash(((JSONObject)o).getString("url"),64).simHash());
					kv.put("anchor", ((JSONObject)o).getString("anchor"));
					kv.put("ext", ((JSONObject)o).getString("ext"));
					kv.put("nofollow", ((JSONObject)o).getString("nofollow"));
					kv.put("link_title", ((JSONObject)o).getString("title"));
					kv.put("type", ((JSONObject)o).getString("type"));
					
					try {
						bw.write(kv.toJSONString());
					}catch (Exception e) {
						
					}
					bw.write("\n");
				}
				System.out.println("Page "+j.getString("url")+" Processed");
			}
			br.close();
			bw.close();
	}
}
